/** @file  fim.c
 *  @brief fim
 *
 */

/*
//|----------------------------------------------------------------------------
//|                 SKY NETWORKS CONFIDENTIAL - COPYRIGHT 2006
//|----------------------------------------------------------------------------
//| The information contained herein is the property of Sky Networks and is
//| strictly confidential. Except as expressly authorized in writing by Sky
//| Networks, the holder shall keep all information contained herein confiden-
//| tial, shall disclose it only to its employees with a need to know, and
//| shall protect it, in whole or in part, from disclosure and dissemination
//| to third parties with the same degree of care it uses to protect its own
//| confidential information of like nature, but with no less than reasonable
//| care. Except expressly authorized in writing by Sky Networks, the holder
//| is granted no rights to use the information contained herein.
//|----------------------------------------------------------------------------
*/

/*
//|----------------------------------------------------------------------------
//| Flexelint
//|----------------------------------------------------------------------------
*/
/*lint -save */
/*lint +libclass(all) */
/* supress the flint warning 534 about return value */
/*lint -esym(534, sprintf, sleep,snprintf,sigemptyset,sigaddset,sigprocmask) */
/*lint -esym(534, read) */
/*
//|----------------------------------------------------------------------------
//| Includes
//|----------------------------------------------------------------------------
*/
#include <stdio.h>
#include <sys/select.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <netinet/in.h>
#include <errno.h>
#include <string.h>
#include <unistd.h>
#include <stdlib.h>
#include <signal.h>
#include <time.h>
#include <sys/ioctl.h>
#include <sys/time.h>
#include <sys/wait.h>
#define  __USE_GNU
#include <pthread.h>
#include <sys/queue.h>
//#include <signal.h>
#include <stdbool.h>
#include <fcntl.h>
#include <sys/resource.h>
#include <libgen.h>
#include <glob.h>
#include "msgid.h"
#include "fim.h"

/*
//|----------------------------------------------------------------------------
//| Defines
//|----------------------------------------------------------------------------
*/
#define NUM_CONFIG_PARAMS      (9)
#define TRACE()                printf("%s",__func__)
#define MAX_ARGS               (63)
#define ALWAYS                 (-1)
#define NEVER                  (-2)

#define DEFAULT_RT_PRIORITY    (1)
#define DEFAULT_NICE           (0)

/*
 * The following defines control how much trap info logging we do.
 */
#define FIM_ALL_TRAP_INFO     (3)
#define FIM_MAX_TRAP_CMD_LEN  (120)

#define CENTRAL_CUR_COREDUMP_DIR "/mnt/usr/coredumps"

#define SEGFAULT_OUTPUT_NAME_PREFIX CENTRAL_CUR_COREDUMP_DIR
#define SEGFAULT_OUTPUT_NAME_ENV    "SEGFAULT_OUTPUT_NAME"
#define SEGFAULT_OUTPUT_NAME_LEN (128)
/*
 * The maximum number of times FIM auto recovers between hard power cycles.
 */
#define MAX_AUTO_REBOOT   ('3')

/* BOOTINFO */
#define BOOTINFO_PATH            "/var/run/bootinfo"
#define BOOTINFO_ACTUAL          BOOTINFO_PATH"/actual"
#define BOOTINFO_REQUESTED       BOOTINFO_PATH"/requested"
#define BOOTINFO_REGION0_STATUS  BOOTINFO_PATH"/region0_status"
#define BOOTINFO_REGION1_STATUS  BOOTINFO_PATH"/region1_status"
#define BOOTINFO_REGION0_STRIKES BOOTINFO_PATH"/region0_strikes"
#define BOOTINFO_REGION1_STRIKES BOOTINFO_PATH"/region1_strikes"

extern int sigignore (int __sig) __THROW;

/*
//-----------------------------------------------------------------------------
//  Static Variables
//-----------------------------------------------------------------------------
*/
static LIST_HEAD(listhead, process_data)  scan_head;
static LIST_HEAD(listhead2, process_data) sanity_head;
static struct fim_options * mv;
static int fdpipe[2];

static int min_rr_priority;
static int max_rr_priority;
static int min_fifo_priority;
static int max_fifo_priority;
static unsigned long int max_affinity_mask;
static const uint32_t max_num_processors = (sizeof(max_affinity_mask)*8);

static int is_fim_dontkill_flag_set(void);
/*
//|--------------------------------------------------------------------------
//| Method     : fim_terminate_processes
//|--------------------------------------------------------------------------
*/
static void fim_terminate_processes(const char * rstring)
{
    char new_rstring[256];
    int sec;
//  enum target_node  node;
    struct process_data * np;
    int status,rc;

//  node = get_target_node();
//  if ((node != node_oam) && (node != node_bh))
//  {
     /* MAC coredumps take a few seconds to finish compressing. */
//     sec = 5;
//     while (sec) {
//       sec = sleep(sec);
//     }

//     sec = 5;
//     while ((sec < 60) && (access(CENTRAL_CUR_COREDUMP_DIR"/.in_progress", F_OK) == 0))
//     {
//        sleep(1);
//        sec++;
//     }
//     printf("FIM SHUTDOWN: Sleep delay completed in %d seconds\n", sec);
//  }

    for( np = scan_head.lh_first; np != 0; np = np->scan_entries.le_next )
    {
        /* Kill everyone but metalog, this will give time for syslog messages to go out */
        if( (np->pid > 0) && strcmp(np->name, "metalog") ) 
        {
            printf("Killing %s (%d)\n", np->name, np->pid);
            (void)kill(np->pid, SIGINT);
        }
    }

    snprintf(new_rstring, 256, "FIM SHUTDOWN: %s\n", rstring);
    printf("%s", new_rstring);

    /* Give everyone time to die */
    sec = 10;
    while (sec) 
    {
        sec = sleep(sec);
    }

    /* Kill everyone still standing, including metalog */
    while( (np = scan_head.lh_first) != 0 )
    {
        if( np->pid > 0 )
        {
            rc = waitpid(np->pid, &status, WNOHANG);
            if( rc != np->pid ) 
            {
                (void)kill(np->pid, SIGKILL);
            }
        }
        LIST_REMOVE(np,scan_entries);
        free(np);
    }

//  breboot(no_strikeout_clear_reset_c ,new_rstring);
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_terminate
//|--------------------------------------------------------------------------
*/
static void fim_terminate(const char * rstring)
{
    printf("Shutting down FIM\n");
    (void)sigignore(SIGCHLD);

    (void)close(mv->socket);
    (void)unlink(mv->fd_path);
    fim_terminate_processes(rstring);
    _exit(-1);
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_readconf
//|--------------------------------------------------------------------------
*/
static int fim_readconf(char const *cfile)
{
    /*lint --e{668} suppress "cfile" is null pass to fopen*/
    char name[20];
    uint32_t  runLevel = 0;
    char restart_string[20];
    char sock_name[80];
    char args[256];
    char * line = NULL;
    char * tmp;
    size_t len = 0;
    int16_t rc = 0;
    struct process_data * item;
    int16_t argnum = 0;
    char * token;
    struct fim_sched fsched;
    unsigned long affinity;
    uint32_t save_stdout;

    FILE *config = fopen(cfile, "r");
    if( config == 0 )
    {
        printf("%s, fopen %s : %s\n", __func__, cfile, strerror(errno));
        return -1;
    }

    while (getline(&line, &len, config) != -1)
    {
        /* Ignore all comments */
        while ( (tmp = strrchr(line,'#')))
            *tmp=0;

        rc = sscanf( line, "%s %d %lx %d %d %s %s %u %[-a-zA-Z0-9_=~:/. $]\n",
                 name, &runLevel, &affinity, &fsched.policy,
                 &fsched.priority, sock_name,restart_string, &save_stdout, args);

        if( rc != NUM_CONFIG_PARAMS )
        {
            if( rc > 0 )
            {
                printf("Error in parsing the config params %d != %d: %s", rc, NUM_CONFIG_PARAMS, line);
            }
            continue;
        }

        item = calloc(1, sizeof(struct process_data));
        if( ! item )
        {
            printf("Out of memory\n");
            return -1;
        }

        /* tokenize the string into args */
        char ** arguments = (char **) calloc(MAX_ARGS+1, sizeof(char *));/*lint !e(429) */
        if( !arguments )
        {
            printf("Out of memory\n");
            free(item);
            return -1;
        }

        argnum = 0;
        char *token_pos;
        token = strtok_r(args, " ", &token_pos);
        while (token != 0 && argnum < MAX_ARGS)
        {
            if( token[0] == '$' )
            {
                //Since the env variable can have spaces in it, we'll separate
                //these into diff args
                if( getenv(&token[1]) )
                {
                    char *arg = strdup(getenv(&token[1]));
                    char *arg_pos;
                    char *env_token = strtok_r(arg, " ", &arg_pos);
                    while(env_token != 0 && argnum < MAX_ARGS)
                    {
                        arguments[argnum++] = strdup(env_token);
                        env_token = strtok_r(0, " ", &arg_pos);
                    }
                    free(arg);
                }
            }
            else
            {
                arguments[argnum++] = strdup(token);
            }

            token = strtok_r(0, " ", &token_pos);
        }
        if( argnum == 0 )
        {
            free(arguments);
            arguments=0;
        }
        else if( argnum == MAX_ARGS )
        {
            arguments[MAX_ARGS-1] = 0;
            printf("Too many arguments for %s\n",name);
        }

        if( !strcmp(restart_string, "A") )
        {
            item->restart_count = ALWAYS;
        }
        else if( ! strcmp(restart_string,"N") )
        {
            item->restart_count = NEVER;
        }
        else
        {
            item->restart_count = strtoull(restart_string, 0, 0);
        }

        item->name = strdup(name);
        if( arguments == NULL )
        {
            printf("the pointer arguments is null \n");
        }
        else
        {
            item->cmd       = arguments[0];
            item->args      = &arguments[0];
        }
        item->run_level = (runLevel >= MAX_RUNLEVEL) ? MAX_RUNLEVEL :
            ((runLevel <= MIN_RUNLEVEL) ? MIN_RUNLEVEL : runLevel);
        item->strikeout = NUM_STRIKEOUT;

        item->save_stdout = save_stdout;

        if( affinity == 0   )
        {
            item->affinity = max_affinity_mask;
            printf("invalid affinity (%lx) for process %s, defaulted to %lx",
                affinity, name, max_affinity_mask);
        }
        else if( affinity > max_affinity_mask )
        {
            item->affinity = max_affinity_mask;
        }
        else
        {
            item->affinity = affinity;
        }


        if( fsched.policy == SCHED_RR )
        {
            if( fsched.priority < min_rr_priority || fsched.priority > max_rr_priority )
            {
                fsched.priority = DEFAULT_RT_PRIORITY;
                printf("invalid priority %d for process %s with policy %d,"
                    " priority defaulted to %d",
                    fsched.priority, name, fsched.policy, fsched.priority);
            }
        }
        else if( fsched.policy == SCHED_FIFO )
        {
            if( fsched.priority < min_fifo_priority || fsched.priority > max_fifo_priority )
            {
                printf("invalid priority %d for process %s with policy %d,"
                    " priority defaulted to %d",
                    fsched.priority, name, fsched.policy, DEFAULT_RT_PRIORITY);

                fsched.priority = DEFAULT_RT_PRIORITY;
            }
        }
        else if( fsched.policy == SCHED_OTHER )
        {
            if( fsched.priority < PRIO_MIN || fsched.priority > PRIO_MAX )
            {
                printf("invalid nice value %d for process %s,"
                    " defaulted to %d",
                    fsched.policy, name, DEFAULT_NICE);

                fsched.priority = DEFAULT_NICE;
            }
        }
        else
        {
            printf("invalid scheduling parameters (%d:%d) for process %s,"
               " defaulted to (SCHED_OTHER:0)",
               fsched.policy, fsched.priority, name);

            fsched.policy = SCHED_OTHER;
            fsched.priority = DEFAULT_NICE;
        }

        item->fsched = fsched;

        if( strcmp(sock_name, "NULL") )
        {
            item->sock_name = strdup(sock_name);
            LIST_INSERT_HEAD(&sanity_head, item, sanity_entries);
        }
        LIST_INSERT_HEAD(&scan_head, item, scan_entries);
    }
  
    if( line )
    {
        free(line);
    }

    (void)fclose(config);

    return 0;
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_init_process
//|--------------------------------------------------------------------------
*/
static int fim_init_process(struct process_data * np)
{
    /*lint --e{613} suppress "np" is null check */
    sigset_t child_sig;
    sigemptyset(&child_sig);
    sigaddset(&child_sig, SIGCHLD);
    sigprocmask(SIG_UNBLOCK, &child_sig, NULL);

    if( (np->pid = fork()) == 0 )
    {
        static char  segfault_name[SEGFAULT_OUTPUT_NAME_LEN+1];
        const char * node_name;

        close(fdpipe[0]);
        close(fdpipe[1]);
        close(mv->socket);

        if( np->save_stdout )
        {
            /*
            * This child will fork once more to pipe app 'np->cmd'
            * stdout/sdterr into logger. Since fim care about the app
            * process signals, the logger process will be the child's
            * child.
            */
#define PIPE_LOGGER 0
#define PIPE_APP    1
            int fds[2];
            pid_t pidLogger=0;

            if( pipe(fds) < 0 )
            {
                printf("Failed to create a pipe: %s\n", strerror(errno));
                return -1;
            }

            if( (pidLogger = fork()) == 0 )
            {
                /*child : logger*/
                char loggerTagOpt[256]; /*match np->args size*/
                char * loggerArgs [/*MAX_ARGS*/] = {
                        (char *)"busybox", (char *)"logger",
                        (char *)"-p" , (char *)"3",
                        (char *)"-t", (char *)loggerTagOpt,
                        NULL};
                char * loggerCmd = loggerArgs[0];

                memset(loggerTagOpt, '\0', sizeof(loggerTagOpt));
                snprintf(loggerTagOpt, 100, "%s_stdout", basename((char *)np->cmd));

                close(fds[PIPE_APP]);
                if( dup2(fds[PIPE_LOGGER], 0) < 0 )  /*stdin*/
                {
                    printf("Failed to redir the inpout to the pipe stdin \n");
                }

                if( execvp(loggerCmd, loggerArgs) < 0 )
                {
                    printf("Failed to exec %s: %s\n", loggerArgs[1], strerror(errno));
                    return -1;
                }
            }

            if( pidLogger > 0 )
            {
                /*this parent is the child of fim, will launch appl*/
                close(fds[PIPE_LOGGER]);
                if( dup2(fds[PIPE_APP], 1) < 0 )  /*stdout*/
                {
                    printf("Failed to redir the output to the pipe stdout \n");
                }
                if( dup2(fds[PIPE_APP], 2) < 0 ) /*stderr*/
                {
                    printf("Failed to redir the output to the pipe stderr \n");
                }
            }
            else
            {
                printf("Child process logger failed to fork from child process %s\n",
                    np->cmd);
                return -1;
            }
        }

        node_name = getenv("NODENAME");
        if( !node_name )
        {
            node_name="node";
        }

        if( snprintf(segfault_name, SEGFAULT_OUTPUT_NAME_LEN, "%s/%s.%d.%s.segv",
                 SEGFAULT_OUTPUT_NAME_PREFIX, node_name, getpid(), np->name) >
                 SEGFAULT_OUTPUT_NAME_LEN )
        {
            printf("segfault_name: %s truncated", segfault_name);
        }
        segfault_name[SEGFAULT_OUTPUT_NAME_LEN]=0;

        if( setenv(SEGFAULT_OUTPUT_NAME_ENV, segfault_name, 1) == -1 )
        {
            printf("setenv failed: %s :%m", segfault_name);
        }

        if( execv(np->cmd, np->args) < 0 )
        {
            printf("Failed to exec %s: %s\n", np->cmd, strerror(errno));
            exit (1);
        }

        if( unsetenv(SEGFAULT_OUTPUT_NAME_ENV) == -1 )
        {
            printf("unsetenv failed: %s :%m", segfault_name);
        }
    }

    sigprocmask(SIG_BLOCK, &child_sig, NULL);

    if( np->pid > 0 )
    {
        np->strikeout = NUM_STRIKEOUT;
        printf("Starting child process for %s : %s with pid %d\n",
             np->name, np->cmd, np->pid);

        // set affinity
        if( sched_setaffinity(np->pid, sizeof(np->affinity), (cpu_set_t*)&(np->affinity)) )
        {
            printf("Failed to set affinity for process %s: %m", np->cmd);
        }

        if( np->fsched.policy == SCHED_OTHER )
        {   /* nice value, should we do PRIO_PGRP instead? */
            if( 0 != setpriority(PRIO_PROCESS, np->pid, np->fsched.priority) )
            {
                printf("Failed to set nice value for process %s: %m", np->cmd);
            }
        }
        else
        {   /* rt priority */
            struct sched_param param = { np->fsched.priority };

            if( 0 != sched_setscheduler(np->pid, np->fsched.policy, &param) )
            {
                printf("Failed to set priority for process %s: %m", np->cmd);
            }
        }
    }
    else
    {
        printf("Child process %s failed to fork", np->cmd);
        return -1;
    }
    return 0;
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_init_runlevel
//|--------------------------------------------------------------------------
*/
static void fim_init_runlevel(uint16_t run_level)
{
    struct process_data * np;

    for (np = scan_head.lh_first; np != 0; np = np->scan_entries.le_next)
    {
        if (np->run_level == run_level)
        {
            if (fim_init_process(np))
            {
                if (is_fim_dontkill_flag_set())
                {
                    printf("init process *%s* failed, Debug flag set: (/.config/.fim_dontkill || --dontkill), no action taken\n", np->name);
                }
                else
                {   
                    printf("init process *%s* failed, terminating fim\n", np->name);
                    fim_terminate("fim_init_runlevel fail");
                }
            }
        }
    }
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_init_up_to_runlevel
//|--------------------------------------------------------------------------
*/
static void fim_init_up_to_runlevel(uint16_t stop_level)
{
    for( ; mv->current_level <= stop_level; mv->current_level++ )
    {
        fim_init_runlevel(mv->current_level);
    }
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_rx_sanity
//|--------------------------------------------------------------------------
*/
static void fim_rx_sanity(void * buffer, char const * sock_name)
{
    /*lint --e{613} suppress "sanityMessage" is null check */
    /*lint --e{668} suppress "sock_name" is null check */
    int found = 0;
    struct process_data * np;
    struct msgHeaderNt *sanityMessage = (struct msgHeaderNt *)buffer;

    if( sanityMessage->control & MSG_CONTROL_NACK )
    {
        printf("Nacked sanity msg for %s\n",sock_name);
        return;
    }
    for (np = sanity_head.lh_first; np != 0; np = np->sanity_entries.le_next)
    {
        if( strcmp(sock_name, np->sock_name) == 0 )
        {
            np->strikeout = NUM_STRIKEOUT;
            (void)gettimeofday(&np->sanityTimestamp,0);
            found = 1;
            break;
        }
    }
    if( !found )
    {
        printf("received unexpected sanity response from %s\n",sock_name);
    }
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_send_status
//|--------------------------------------------------------------------------
*/
static int fim_send_status(char const * sock_name)
{
    struct msgHeaderNt header;
    struct fimStatusNt stmsg;
    struct process_data * np;
    int slen;

    msg_control_set(&stmsg.header.control,MSG_CONTROL_ACK);
    stmsg.header.type = FIM_STATUS;
    stmsg.header.length = sizeof(struct fimStatusNt);
    stmsg.header.sequence = 0;
    stmsg.header.sequenceReply = 0;
    (void)gettimeofday(&stmsg.header.txTime,0);

    slen = sprintf(stmsg.statusString, "Fim Run Level is: %d", mv->current_level-1);
    stmsg.header.length = sizeof(struct fimStatusNt) + slen;
    if( fim_send_to(mv->socket,sock_name, &stmsg, stmsg.header.length,0) )
    {
        printf("%s\n", "Failed to send status end message.");
        return -1;
    }

#define FMT "%-25s %-15s %10s %-5s %16s"
    slen = sprintf(stmsg.statusString, FMT,
               "Name", "Socket", "Run Level", "Pid", "Sanity Timestamp");
    stmsg.header.length = sizeof(struct fimStatusNt) + slen;
    if( fim_send_to(mv->socket, sock_name, &stmsg, stmsg.header.length, 0) )
    {
        printf("%s\n", "Failed to send status end message.");
        return -1;
    }

    slen = sprintf(stmsg.statusString, FMT,
               "----", "------", "---------", "---", "----------------");
    stmsg.header.length = sizeof(struct fimStatusNt) + slen;
    if( fim_send_to(mv->socket,sock_name, &stmsg, stmsg.header.length, 0) )
    {
        printf("%s\n", "Failed to send status end message.");
        return -1;
    }

    for( np = scan_head.lh_first; np != 0; np = np->scan_entries.le_next )
    {
#undef FMT
#define FMT "%-25s %-15s %10d %-5d %16u"
        slen=sprintf(stmsg.statusString, FMT,
                 np->name,
                 np->sock_name,
                 np->run_level,
                 np->pid,
                 (unsigned int)(np->sanityTimestamp.tv_sec));
        stmsg.header.length = sizeof(struct fimStatusNt) + slen;
        if( fim_send_to(mv->socket,sock_name, &stmsg, stmsg.header.length, 0) )
        {
            printf("%s\n", "Failed to send status end message.");
            return -1;
        }
    }

    header.type          = FIM_STATUS_END;
    header.length        = sizeof(struct msgHeaderNt);
    header.sequence      = 0;
    header.sequenceReply = 0;
    (void)gettimeofday(&header.txTime, 0);

    if( fim_send_to(mv->socket,sock_name,&header,header.length, 0) )
    {
        printf("%s\n", "Failed to send status end message.");
        return -1;
    }

    return 0;
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_run_level_msg
//|--------------------------------------------------------------------------
*/
void fim_run_level_msg
(
    void * bufferPtr,
    const struct sockaddr_un * remoteAddr
)
{
    struct fimRunLevelNt *message = (struct fimRunLevelNt *)bufferPtr;
    if( message == NULL )
    {
        printf("the message buffer  is null here \n");
        return;
    }

    fim_init_up_to_runlevel(message->runLevel);
    msg_control_set(&message->header.control, MSG_CONTROL_ACK);

    printf("FIM runlevel control: %d\n",message->header.control);
    if( remoteAddr == NULL )
    {
        printf("the remoteAddr is null here \n");
        return;
    }
    (void)fim_send_to(mv->socket,remoteAddr->sun_path, message, message->header.length, 0);
}

/*
//|--------------------------------------------------------------------------
//| Method     : has_fim_reached_max_auto_recovery
//|
//| description: This method checks if FIM should kill it's child processes
//|              and reboot the modem. It reads the strikeout counters
//|              in both SRAM banks. If they are both at or less than the
//|              maximum then it reboots the modem.
//|
//| returns:     0 if modem should be rebooted
//|              1 if FIM takes no action
//|--------------------------------------------------------------------------
*/
static int has_fim_reached_max_auto_recovery(void)
{
    FILE    *         zoneStrikeoutFile;
    char              zoneRegion0StrikeOutCnt = 0;
    char              zoneRegion1StrikeOutCnt = 0;
    int               rc = 0;            /* means FIM will kill children */


    /* read strikeout count in zone 0 */
    if( (zoneStrikeoutFile = fopen(BOOTINFO_REGION0_STRIKES, "r")) == 0 )
    {
        printf("FIM Could not open %s, errno %d", BOOTINFO_REGION0_STRIKES, errno);
        return 0;
    }
    /* read the current count */
    zoneRegion0StrikeOutCnt = fgetc(zoneStrikeoutFile);
    fclose(zoneStrikeoutFile);

    /* read strikeout count in zone 1 */
    if( (zoneStrikeoutFile = fopen(BOOTINFO_REGION1_STRIKES, "r")) == 0 )
    {
        printf("FIM Could not open %s, errno %d", BOOTINFO_REGION1_STRIKES, errno);
        return 0;
    }
    /* read the current count */
    zoneRegion1StrikeOutCnt = fgetc(zoneStrikeoutFile);
    fclose(zoneStrikeoutFile);

    /* check if we auto reboot the modem */
    if( zoneRegion0StrikeOutCnt>= MAX_AUTO_REBOOT &&
        zoneRegion1StrikeOutCnt>= MAX_AUTO_REBOOT )
    {
        /* flag that we don't want auto recovery */
        rc = 1;

        /* only log this once */
        if( zoneRegion0StrikeOutCnt == MAX_AUTO_REBOOT &&
            zoneRegion1StrikeOutCnt == MAX_AUTO_REBOOT )
        {
            printf("****FIM STOPPING AUTO RECOVERY auto recovery from fault exceeded****");
        }
    }

    return rc;
} /* has_fim_reached_max_auto_recovery */

/*
//|--------------------------------------------------------------------------
//| Method     : is_fim_dontkill_flag_set
//|--------------------------------------------------------------------------
*/
static int is_fim_dontkill_flag_set(void)
{
    int rv = access("/.config/.fim_dontkill",F_OK);

    if( (rv == 0) || (mv->dontkill == true) )
    {
        return 1;
    }

    /* check if FIM has reached maximum number recovery attempts */
    if( has_fim_reached_max_auto_recovery() )
    {
        /* don't reboot the modem */
        return 1;
    }

    return 0;
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_take_action
//|--------------------------------------------------------------------------
*/
static void fim_take_action
(
    struct process_data * np,
    int status
)
{
#define BUF_SIZE 256
    bool signignore=false;
    char statbuf[BUF_SIZE];

    if( np == NULL )
    {
        printf("the struct of the process_data:np is null here \n");
        return;
    }

    statbuf[0] = 0;
    if( WIFEXITED(status) )
    {
        snprintf(statbuf, BUF_SIZE, "%s EXITED with status %d", np->name, WEXITSTATUS(status));
    }
    else if( WIFSIGNALED(status) )
    {
        snprintf(statbuf, BUF_SIZE, "%s SIGNALED with status %d", np->name, WTERMSIG(status));
        if( (WTERMSIG(status) == 9) || (WTERMSIG(status) == 15) )
        {
            printf("No action taken child %s signalled with (KILLSIG or SIGTERM)", statbuf);
            signignore=true;
        }
    }
    else if( WIFSTOPPED(status) )
    {
        snprintf(statbuf, BUF_SIZE, "%s STOPPED with status %d", np->name, WSTOPSIG(status));
    }

    if( signignore )
    {
        printf("Ignoring signal : %s\n", statbuf);
    }
    else if( np->restart_count == NEVER )
    {
        printf("%s, not restarting\n", statbuf);
    }
    else if( is_fim_dontkill_flag_set() == 1 )
    {
        printf("At reboot limit or debug flag set(/.config/.fim_dontkill||--dontkill): %s, no action taken", statbuf);
    }
    else if( np->num_restarted == np->restart_count )
    {
        printf("%s\n", statbuf);
        /* We must restart the card, for now, just shutdown everyone.  This will
         * kill eec and the wdog will reset the card
         */
        fim_terminate(statbuf);
    }
    else
    {
        /* Restart it */
        if( np->restart_count != ALWAYS )
        {
            np->num_restarted++;
        }
        printf("%s, restart # %d\n", statbuf, np->num_restarted);
        if( fim_init_process(np) )
        {
            fim_terminate(statbuf);
        }
    }
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_scan_child
//|--------------------------------------------------------------------------
*/
void fim_scan_child(void)
{
    struct process_data * np;
    int status;
    pid_t pid;
    struct timeval timeLogStart;
    struct timeval timeLogEnd;
    char  trapinfo[FIM_MAX_TRAP_CMD_LEN];

    while( (pid = waitpid(-1,&status,WNOHANG)) > 0 )
    {
        for( np = scan_head.lh_first; np != 0; np = np->scan_entries.le_next )
        {
            if( np->pid == pid )
            {
                if( WIFEXITED(status) &&
                    WEXITSTATUS(status) == 0 &&
                    (np->restart_count == NEVER) )
                {
                    // Don't trap when the process exit gracefully
                    // with restart_count stated to NEVER
                }
                else
                {
                    /* get time stamp before logging */
                    (void)gettimeofday(&timeLogStart,0);

                    /* construct argument string for system call */
                    snprintf(trapinfo, FIM_MAX_TRAP_CMD_LEN,
                        "/usr/bin/trapinfo.sh %d %d %s '%s'",
                        FIM_ALL_TRAP_INFO, pid,
                        np->name, np->cmd);

                    /* collect the trap info */
                    if( system(trapinfo) == -1 )
                    {
                        printf("trapinfo command is failed \n");
                    }

                    /* get time stamp at end of logging */
                    (void)gettimeofday(&timeLogEnd,0);
                    printf("TRAP PID (%d) log start sec (%d) usec (%d) log end sec (%d) usec (%d)\n",
                        pid,
                        (uint32_t)timeLogStart.tv_sec, (uint32_t)timeLogStart.tv_usec,
                    (uint32_t)timeLogEnd.tv_sec, (uint32_t)timeLogEnd.tv_usec);
                }

                /* Does it need restarting or do we reboot? */
                fim_take_action(np, status);
            }
        }
    }
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_do_sanity
//|--------------------------------------------------------------------------
*/
void fim_do_sanity(void)
{
    struct msgHeaderNt header;
    struct process_data * np;

    header.type = 99;
    header.length = sizeof(struct msgHeaderNt);
    header.sequence = 0;
    header.sequenceReply = 0;

    for( np = sanity_head.lh_first; np != 0; np = np->sanity_entries.le_next )
    {
        if( np->sock_name )
        {
            if( np->run_level >= mv->current_level )
            {
                printf("Not sending sanity message to '%s'. Current run level is %d",
                    np->name, mv->current_level-1);
                continue;
            }
            (void)gettimeofday(&header.txTime, 0);
            if( !np->strikeout-- )
            {
                /* No response from last few messages, take action */
                fim_take_action(np, 0);
                continue;
            }
            (void)fim_send_to(mv->socket,np->sock_name,&header,header.length, 0);
        }
    }
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_sigchld_handler
//|--------------------------------------------------------------------------
*/
static void  fim_sigchld_handler(int no __attribute__((unused)),
                                 siginfo_t * ptr __attribute__((unused)),
                                 void *ptr2 __attribute__((unused)))
{
    char dummy;
    /* Read data from rpipe and then write it to wpipe. */
    /* The rpipe could be empty, and the actual value of read data in dummy 
       buffer is not cared here. Just write any character to wpipe.*/
    read(fdpipe[0], &dummy, sizeof(dummy));
    if( write(fdpipe[1], &dummy, sizeof(dummy)) == -1 )
    {
        printf("Write the pipe failed \n");
    }
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_process_messages
//|--------------------------------------------------------------------------
*/
static int fim_process_messages(char * buffer)
{
    struct sockaddr_un rx_addr;
    struct msgHeaderNt *message;
    uint32_t cc;
    int rc;
    fd_set read_fds;
    struct timeval tv;
    sigset_t   child_sig;
    int max_fd;

    if( pipe(fdpipe) )
    {
        printf("pipe failed with error %s\n", strerror(errno));
        exit(0);
    }
    int rpipe = fdpipe[0];

    int flags = fcntl(rpipe, F_GETFL, 0);
    if( fcntl(rpipe, F_SETFL, flags | O_NDELAY) == -1 )
    {
        printf("changed controling of the open pipe failed \n");
    }

    message = (struct msgHeaderNt *)buffer;
    if( message == NULL )
    {
        printf("the message buffer is null \n");
        exit(0);
    }

    struct sigaction action;
    sigemptyset(&action.sa_mask);
    action.sa_sigaction  = fim_sigchld_handler;
    action.sa_flags      = SA_SIGINFO;

    if( sigaction(SIGCHLD, &action, NULL) )
    {
        printf("sigaction failed with error %s\n", strerror(errno));
        exit(0);
    }
    sigemptyset(&child_sig);
    sigaddset(&child_sig, SIGCHLD);
    sigprocmask(SIG_BLOCK, &child_sig, 0);

    /* To clear current rpipe before starting any child process */
    /* The rpipe could be empty, and the read out buffer is not used. */
    read(rpipe, buffer, MSG_MAX_LEN);

    fim_init_up_to_runlevel(mv->start_level);

    max_fd = ((rpipe>mv->socket) ? rpipe : mv->socket) + 1;
    for( ; ; )
    {
        sigprocmask(SIG_UNBLOCK, &child_sig, NULL);

        /* It should be safer to set file descriptors and struct timeval for each loop. */
        do
        {
            FD_ZERO(&read_fds);
            FD_SET(mv->socket,&read_fds);
            FD_SET(rpipe,&read_fds);

            sigprocmask(SIG_UNBLOCK, &child_sig, NULL);
            tv.tv_sec = SANITY_TIMER_SEC; tv.tv_usec = 0;
        }
        while( ((rc = select(max_fd,&read_fds,0,0,&tv)) == -1) &&
               (errno == EINTR) );

        sigprocmask(SIG_BLOCK, &child_sig, NULL);
        if( rc == -1 )
        {
            char serror[256];
            snprintf(serror, 256, "select failed : %s\n", strerror(errno));
            printf("%s", serror);
            if( !is_fim_dontkill_flag_set() )
            {
                fim_terminate(serror);
            }
        }

        if( rc == 0 )
        {
            /* Timeout */
            fim_do_sanity();
            continue;
        }

        if( FD_ISSET(rpipe, &read_fds) )
        {
            /* Got SIGCHLD */
            if( read(rpipe, buffer, MSG_MAX_LEN) == -1 )
            {
                printf("Read the pipe failed \n");
            }
            fim_scan_child();
        }
        if( !FD_ISSET(mv->socket, &read_fds) ) continue;

        /* Got messages */
        socklen_t rx_addr_len = sizeof(struct sockaddr_un);
        cc = recvfrom(mv->socket, buffer, MSG_MAX_LEN, 0,
                   (struct sockaddr*)((void *)&rx_addr), &rx_addr_len );
        if( cc <= 0 )
        {
            char serror[256];
            snprintf(serror, 256, "recvfrom failed : %s\n", strerror(errno));
            printf("%s", serror);
            fim_terminate(serror);
        }

        switch( message->type )
        {
            case FIM_DONTKILL:
            {
                msg_control_set(&message->control,MSG_CONTROL_ACK);
                mv->dontkill = true;
                (void)fim_send_to(mv->socket, rx_addr.sun_path, message, message->length, 0);
                break;
            }
            case MSG_COMMON_SANITY:
            {
                fim_rx_sanity(buffer,rx_addr.sun_path);
                break;
            }

            case FIM_STATUS:
            {
                (void)fim_send_status(rx_addr.sun_path);
                break;
            }

            case FIM_TERMINATE:
            {
                msg_control_set(&message->control, MSG_CONTROL_ACK);
                (void)fim_send_to(mv->socket, rx_addr.sun_path, message, message->length, 0);
                fim_terminate("FIM terminated by user");
                break;
            }
            case FIM_RUNLEVEL:
            {
                fim_run_level_msg( buffer, &rx_addr);
                break;
            }
            default:
            {
                printf("Unknown message (%d) received on %d from: %s\n",
                    message->type, mv->socket, rx_addr.sun_path);
                break;
            }
        }
    }
    free(buffer);
    return 0;
}

/*
//|--------------------------------------------------------------------------
//| Method     : fim_main
//|--------------------------------------------------------------------------
*/
int fim_main(struct fim_options * in)
{
    void *buffer;
    int rc;
    struct rlimit lim;
    glob_t globbuf;
    size_t cfile;

    if( !in )
        return 0;

    mv = in;

    printf("FIM starting level %d ...\n", mv->start_level);

    /* Up rlimit for core dumps */
    if( getrlimit(RLIMIT_CORE,&lim) )
    {
        printf("getrlimit %m\n");
        return -1;
    }

#define CORE_DUMPS_MAX (mv->corelimit)
    lim.rlim_cur = (CORE_DUMPS_MAX < lim.rlim_max) ? CORE_DUMPS_MAX : lim.rlim_max;

    if( setrlimit(RLIMIT_CORE, &lim) )
    {
        printf("setrlimit %m\n");
        return -1;
    }

    /* Determine the actual number of processors */
    int num_procs = sysconf(_SC_NPROCESSORS_CONF);
    if( num_procs == -1 || (unsigned int)num_procs > max_num_processors )
    {
        max_affinity_mask = ~max_affinity_mask;
        printf("unable to determine number of cores, max_affinity_mask default to %lx",
             max_affinity_mask);
    }
    else
    {
        max_affinity_mask = (~((unsigned long long)0) >> (sizeof(unsigned long long)*8-num_procs));
    }

    /* Determine min and max rt priorities */
    min_rr_priority = sched_get_priority_min(SCHED_RR);
    max_rr_priority = sched_get_priority_max(SCHED_RR);
    min_fifo_priority = sched_get_priority_min(SCHED_FIFO);
    max_fifo_priority = sched_get_priority_max(SCHED_FIFO);

    if( min_rr_priority == -1 )
    {
        min_rr_priority = DEFAULT_RT_PRIORITY;
        printf("error getting min_rr_priority, default to %d\n", min_rr_priority);
    }

    if( max_rr_priority == -1 )
    {
        max_rr_priority = DEFAULT_RT_PRIORITY;
        printf("error getting max_rr_priority, default to %d\n", max_rr_priority);
    }

    if( min_fifo_priority == -1 )
    {
        min_fifo_priority = DEFAULT_RT_PRIORITY;
        printf("error getting min_fifo_priority, default to %d\n", min_fifo_priority);
    }

    if( max_fifo_priority == -1 )
    {
        max_fifo_priority = DEFAULT_RT_PRIORITY;
        printf("error getting max_fifo_priority, default to %d\n", max_fifo_priority);
    }

    LIST_INIT(&scan_head);
    LIST_INIT(&sanity_head);

    if( (mv->socket = fim_create_socket(mv->fd_path)) == 0 )
    {
        printf("%s\n","Fim failed to initialize sockets\n");
        return -1;
    }

printf("%s, hydali 2, %s \n", __func__, mv->cfile);

    char *token;
    //we use strtok_r with 'pos' since strtok is called from function fim_readconf
    char *pos;
    int cfNum;
    for (token = strtok_r(mv->cfile, ",", &pos), cfNum = 1; token != 0; token = strtok_r(0, ",", &pos))
    {
        printf("Fim config glob string=%s\n", token);

        /* Find glob config file names */
        if( glob(token, 0, NULL, &globbuf) != 0 )
        {
            printf("Fim failed to glob config files\n");
            return -1;
        }

        printf("Found %d Fim config file\n", globbuf.gl_pathc);

        for (cfile = 0; cfile < globbuf.gl_pathc; cfile++)
        {
            printf("Fim config file[%d]=%s\n", cfNum++, globbuf.gl_pathv[cfile]);

            /* Read the fd configuration data */
            if( fim_readconf(globbuf.gl_pathv[cfile]) )
            {
                printf("Fim failed to read config files\n");
                return -1;
            }
        }

        /* Done with the glob data structure, free it */
        globfree( &globbuf );
    }

    if( posix_memalign(&buffer, 32, MSG_MAX_LEN) != 0 )
    {
        printf("Could not allocate buffer <%d>: %s\n", errno, strerror(errno));
        return -1;
    }

    rc = fim_process_messages(buffer);
    free(buffer);
    return rc;
}
